{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MMM\n",
      "ABT\n",
      "ABBV\n",
      "ACN\n",
      "ATVI\n",
      "AYI\n",
      "ADBE\n",
      "AMDAAP\n",
      "\n",
      "AES\n",
      "AET\n",
      "AMG\n",
      "AFL\n",
      "A\n",
      "APD\n",
      "AKAM\n",
      "ALK\n",
      "ALB\n",
      "ARE\n",
      "ALXN\n",
      "ALGN\n",
      "ALLE\n",
      "AGN\n",
      "ADS\n",
      "LNTALL\n",
      "\n",
      "GOOGL\n",
      "GOOG\n",
      "MOAMZN\n",
      "\n",
      "AEE\n",
      "AAL\n",
      "AEP\n",
      "AXP\n",
      "AIG\n",
      "AMT\n",
      "AWK\n",
      "AMP\n",
      "ABC\n",
      "AMEAMGN\n",
      "\n",
      "APH\n",
      "APC\n",
      "ADI\n",
      "ANDV\n",
      "ANSS\n",
      "ANTM\n",
      "AON\n",
      "AOS\n",
      "APA\n",
      "AIV\n",
      "AAPL\n",
      "bad: APC\n",
      "AMATAPTV\n",
      "\n",
      "ADM\n",
      "ARNC\n",
      "AJG\n",
      "AIZ\n",
      "T\n",
      "ADSK\n",
      "ADP\n",
      "AZO\n",
      "AVB\n",
      "AVY\n",
      "BHGE\n",
      "BLL\n",
      "BAC\n",
      "BK\n",
      "BAX\n",
      "BBT\n",
      "BDX\n",
      "BRK.B\n",
      "BBY\n",
      "BIIB\n",
      "BLK\n",
      "HRB\n",
      "BA\n",
      "BWA\n",
      "bad: BRK.B\n",
      "BXP\n",
      "BSX\n",
      "BHF\n",
      "BMY\n",
      "AVGO\n",
      "BF.B\n",
      "CHRW\n",
      "CA\n",
      "COG\n",
      "CDNS\n",
      "CPB\n",
      "COF\n",
      "CAH\n",
      "bad: BF.B\n",
      "CBOE\n",
      "KMX\n",
      "CCL\n",
      "CAT\n",
      "CBG\n",
      "CBS\n",
      "CELG\n",
      "CNC\n",
      "CNP\n",
      "CTL\n",
      "CERN\n",
      "CF\n",
      "SCHWbad: CBG\n",
      "\n",
      "CHTR\n",
      "CHK\n",
      "CVX\n",
      "CMG\n",
      "CB\n",
      "CHD\n",
      "CI\n",
      "XEC\n",
      "CINF\n",
      "CTAS\n",
      "CSCO\n",
      "C\n",
      "CFG\n",
      "CTXS\n",
      "CLX\n",
      "CME\n",
      "CMS\n",
      "KO\n",
      "CTSH\n",
      "CL\n",
      "CMCSA\n",
      "CMA\n",
      "CAG\n",
      "CXO\n",
      "COP\n",
      "ED\n",
      "STZ\n",
      "COO\n",
      "GLW\n",
      "COST\n",
      "COTY\n",
      "CCI\n",
      "CSRA\n",
      "CSX\n",
      "CMI\n",
      "CVS\n",
      "DHI\n",
      "DHR\n",
      "DRI\n",
      "DVA\n",
      "DE\n",
      "DAL\n",
      "XRAYDVN\n",
      "\n",
      "DLR\n",
      "DFS\n",
      "DISCA\n",
      "DISCK\n",
      "DISH\n",
      "DG\n",
      "DLTR\n",
      "D\n",
      "DOV\n",
      "DWDP\n",
      "DPS\n",
      "DTE\n",
      "DRE\n",
      "DUKDXC\n",
      "\n",
      "ETFC\n",
      "EMN\n",
      "ETN\n",
      "EBAY\n",
      "bad: DPS\n",
      "ECL\n",
      "bad: DWDP\n",
      "EIX\n",
      "EW\n",
      "EA\n",
      "EMR\n",
      "ETR\n",
      "EVHC\n",
      "EOG\n",
      "EQT\n",
      "EFX\n",
      "EQIX\n",
      "EQR\n",
      "ESS\n",
      "EL\n",
      "ES\n",
      "RE\n",
      "EXC\n",
      "EXPE\n",
      "EXPD\n",
      "ESRX\n",
      "EXR\n",
      "XOM\n",
      "FFIV\n",
      "FB\n",
      "FAST\n",
      "FRT\n",
      "FDX\n",
      "FIS\n",
      "FITB\n",
      "FE\n",
      "FISV\n",
      "FLIR\n",
      "FLS\n",
      "FLR\n",
      "FMC\n",
      "FL\n",
      "F\n",
      "FTV\n",
      "FBHS\n",
      "BEN\n",
      "FCX\n",
      "GPS\n",
      "GRMN\n",
      "IT\n",
      "GD\n",
      "GE\n",
      "GGP\n",
      "GIS\n",
      "GM\n",
      "GPCbad: GGP\n",
      "\n",
      "GILD\n",
      "GPN\n",
      "GS\n",
      "GT\n",
      "GWW\n",
      "HAL\n",
      "HBI\n",
      "HOG\n",
      "HRS\n",
      "HIG\n",
      "HAS\n",
      "HCA\n",
      "HCP\n",
      "HP\n",
      "bad: HRS\n",
      "HSIC\n",
      "HSY\n",
      "HES\n",
      "HPE\n",
      "HLT\n",
      "HOLX\n",
      "HD\n",
      "HON\n",
      "HRL\n",
      "HST\n",
      "HPQ\n",
      "HUM\n",
      "HBAN\n",
      "HII\n",
      "IDXX\n",
      "INFO\n",
      "ITW\n",
      "ILMN\n",
      "IR\n",
      "INTC\n",
      "ICE\n",
      "IBM\n",
      "INCY\n",
      "IP\n",
      "IPG\n",
      "IFF\n",
      "INTU\n",
      "ISRG\n",
      "IVZ\n",
      "IQV\n",
      "IRM\n",
      "JEC\n",
      "JBHT\n",
      "SJM\n",
      "JNJ\n",
      "JCI\n",
      "JPM\n",
      "JNPR\n",
      "KSU\n",
      "K\n",
      "KEY\n",
      "KMB\n",
      "KIM\n",
      "KMI\n",
      "KLAC\n",
      "KSS\n",
      "KHC\n",
      "KR\n",
      "LB\n",
      "LLL\n",
      "LH\n",
      "LRCX\n",
      "LEG\n",
      "bad: LLL\n",
      "LEN\n",
      "LUK\n",
      "LLY\n",
      "LNC\n",
      "LKQ\n",
      "LMT\n",
      "L\n",
      "bad: LUK\n",
      "LOW\n",
      "LYB\n",
      "MTB\n",
      "MAC\n",
      "M\n",
      "MRO\n",
      "MPC\n",
      "MAR\n",
      "MMC\n",
      "MLM\n",
      "MAS\n",
      "MA\n",
      "MAT\n",
      "MKC\n",
      "MCD\n",
      "MCK\n",
      "MDT\n",
      "MRK\n",
      "MET\n",
      "MTD\n",
      "MGM\n",
      "KORS\n",
      "MCHP\n",
      "MU\n",
      "MSFT\n",
      "bad: KORS\n",
      "MAA\n",
      "MHK\n",
      "TAP\n",
      "MDLZ\n",
      "MON\n",
      "MNST\n",
      "MCO\n",
      "MS\n",
      "MOS\n",
      "MSI\n",
      "MYL\n",
      "NDAQ\n",
      "NOV\n",
      "NAVI\n",
      "NTAP\n",
      "NFLX\n",
      "NWL\n",
      "NFX\n",
      "NEM\n",
      "NWSA\n",
      "NWS\n",
      "NEE\n",
      "NLSN\n",
      "NKE\n",
      "NI\n",
      "NBL\n",
      "JWN\n",
      "NSC\n",
      "NTRS\n",
      "NOC\n",
      "NCLH\n",
      "NRG\n",
      "NUE\n",
      "NVDA\n",
      "ORLY\n",
      "OXY\n",
      "OMC\n",
      "OKE\n",
      "ORCL\n",
      "PCAR\n",
      "PKG\n",
      "PH\n",
      "PDCO\n",
      "PAYX\n",
      "PYPL\n",
      "PNR\n",
      "PBCT\n",
      "PEP\n",
      "PKI\n",
      "PRGO\n",
      "PFE\n",
      "PCG\n",
      "PM\n",
      "PSX\n",
      "PNW\n",
      "PXD\n",
      "PNC\n",
      "RL\n",
      "PPG\n",
      "PPL\n",
      "PX\n",
      "PCLN\n",
      "PFG\n",
      "PGPGR\n",
      "\n",
      "PLD\n",
      "PRU\n",
      "PEG\n",
      "PSA\n",
      "PHM\n",
      "bad: PCLN\n",
      "PVH\n",
      "QRVO\n",
      "PWR\n",
      "QCOM\n",
      "DGX\n",
      "RRC\n",
      "RJF\n",
      "RTN\n",
      "O\n",
      "RHT\n",
      "REG\n",
      "REGN\n",
      "RF\n",
      "RSG\n",
      "RMD\n",
      "RHI\n",
      "ROK\n",
      "COL\n",
      "bad: RHT\n",
      "ROP\n",
      "ROST\n",
      "RCL\n",
      "CRM\n",
      "SBAC\n",
      "SCG\n",
      "SLB\n",
      "SNI\n",
      "STX\n",
      "SEE\n",
      "SRE\n",
      "SHW\n",
      "SIG\n",
      "SPGSWKS\n",
      "\n",
      "SLG\n",
      "SNA\n",
      "SO\n",
      "LUV\n",
      "SPGI\n",
      "SWK\n",
      "SBUX\n",
      "STT\n",
      "SRCL\n",
      "SYK\n",
      "STI\n",
      "SYMC\n",
      "SYF\n",
      "SNPS\n",
      "SYY\n",
      "TROW\n",
      "TPR\n",
      "TGT\n",
      "TEL\n",
      "FTI\n",
      "TXN\n",
      "TXT\n",
      "TMO\n",
      "TIF\n",
      "TWX\n",
      "TJX\n",
      "TMK\n",
      "TSS\n",
      "TSCO\n",
      "TDG\n",
      "TRV\n",
      "TRIP\n",
      "FOXA\n",
      "FOX\n",
      "TSN\n",
      "UDR\n",
      "ULTA\n",
      "USB\n",
      "bad: TMK\n",
      "UAA\n",
      "UA\n",
      "UNP\n",
      "UAL\n",
      "UNH\n",
      "UPS\n",
      "URI\n",
      "UTX\n",
      "UHS\n",
      "UNM\n",
      "VFC\n",
      "VLO\n",
      "VAR\n",
      "VTR\n",
      "VRSN\n",
      "VRSK\n",
      "VZ\n",
      "VRTX\n",
      "VIAB\n",
      "V\n",
      "VNO\n",
      "VMC\n",
      "WMT\n",
      "WBA\n",
      "DIS\n",
      "WM\n",
      "WAT\n",
      "WEC\n",
      "WFC\n",
      "HCN\n",
      "WDC\n",
      "WU\n",
      "WRK\n",
      "WY\n",
      "WHR\n",
      "WMB\n",
      "WLTW\n",
      "WYN\n",
      "WYNN\n",
      "XEL\n",
      "bad: HCN\n",
      "XRX\n",
      "XLNX\n",
      "XL\n",
      "bad: WYN\n",
      "XYL\n",
      "YUM\n",
      "ZBH\n",
      "bad: XL\n",
      "ZION\n",
      "ZTS\n",
      "getSandP_threaded.py\n",
      "The threaded script took 1 minutes and 5 seconds to run.\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "from concurrent import futures\n",
    "\n",
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "import pandas_datareader.data as web\n",
    "\n",
    "\n",
    "def download_stock(stock):\n",
    "\n",
    "\"\"\" try to query the iex for a stock, if failed note with print \"\"\"\n",
    "\n",
    "    try:\n",
    "        print(stock)\n",
    "        stock_df = web.DataReader(stock,'yahoo', start_time, now_time)\n",
    "        stock_df['Name'] = stock\n",
    "        output_name = stock + '_data.csv'\n",
    "        stock_df.to_csv(output_name)\n",
    "\n",
    "    except:\n",
    "        bad_names.append(stock)\n",
    "        print('bad: %s' % (stock))\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "\"\"\" set the download window \"\"\"\n",
    "\n",
    "    now_time = datetime.now()\n",
    "    start_time = datetime(now_time.year - 5, now_time.month , now_time.day)\n",
    "\n",
    "    \"\"\" list of sp500 companies \"\"\"\n",
    "\n",
    "    s_and_p = ['MMM','ABT','ABBV','ACN','ATVI','AYI','ADBE','AMD','AAP','AES','AET',\n",
    "\n",
    "\t\t'AMG','AFL','A','APD','AKAM','ALK','ALB','ARE','ALXN','ALGN','ALLE',\n",
    "\n",
    "\t\t'AGN','ADS','LNT','ALL','GOOGL','GOOG','MO','AMZN','AEE','AAL','AEP',\n",
    "\n",
    "\t\t'AXP','AIG','AMT','AWK','AMP','ABC','AME','AMGN','APH','APC','ADI','ANDV',\n",
    "\n",
    "\t\t'ANSS','ANTM','AON','AOS','APA','AIV','AAPL','AMAT','APTV','ADM','ARNC',\n",
    "\n",
    "\t\t'AJG','AIZ','T','ADSK','ADP','AZO','AVB','AVY','BHGE','BLL','BAC','BK',\n",
    "\n",
    "\t\t'BAX','BBT','BDX','BRK.B','BBY','BIIB','BLK','HRB','BA','BWA','BXP','BSX',\n",
    "\n",
    "\t\t'BHF','BMY','AVGO','BF.B','CHRW','CA','COG','CDNS','CPB','COF','CAH','CBOE',\n",
    "\n",
    "\t\t'KMX','CCL','CAT','CBG','CBS','CELG','CNC','CNP','CTL','CERN','CF','SCHW',\n",
    "\n",
    "\t\t'CHTR','CHK','CVX','CMG','CB','CHD','CI','XEC','CINF','CTAS','CSCO','C','CFG',\n",
    "\n",
    "\t\t'CTXS','CLX','CME','CMS','KO','CTSH','CL','CMCSA','CMA','CAG','CXO','COP',\n",
    "\n",
    "\t\t'ED','STZ','COO','GLW','COST','COTY','CCI','CSRA','CSX','CMI','CVS','DHI',\n",
    "\n",
    "\t\t'DHR','DRI','DVA','DE','DAL','XRAY','DVN','DLR','DFS','DISCA','DISCK','DISH',\n",
    "\n",
    "\t\t'DG','DLTR','D','DOV','DWDP','DPS','DTE','DRE','DUK','DXC','ETFC','EMN','ETN',\n",
    "\n",
    "\t\t'EBAY','ECL','EIX','EW','EA','EMR','ETR','EVHC','EOG','EQT','EFX','EQIX','EQR',\n",
    "\n",
    "\t\t'ESS','EL','ES','RE','EXC','EXPE','EXPD','ESRX','EXR','XOM','FFIV','FB','FAST',\n",
    "\n",
    "\t\t'FRT','FDX','FIS','FITB','FE','FISV','FLIR','FLS','FLR','FMC','FL','F','FTV',\n",
    "\n",
    "\t\t'FBHS','BEN','FCX','GPS','GRMN','IT','GD','GE','GGP','GIS','GM','GPC','GILD',\n",
    "\n",
    "\t\t'GPN','GS','GT','GWW','HAL','HBI','HOG','HRS','HIG','HAS','HCA','HCP','HP','HSIC',\n",
    "\n",
    "\t\t'HSY','HES','HPE','HLT','HOLX','HD','HON','HRL','HST','HPQ','HUM','HBAN','HII',\n",
    "\n",
    "\t\t'IDXX','INFO','ITW','ILMN','IR','INTC','ICE','IBM','INCY','IP','IPG','IFF','INTU',\n",
    "\n",
    "\t\t'ISRG','IVZ','IQV','IRM','JEC','JBHT','SJM','JNJ','JCI','JPM','JNPR','KSU','K','KEY',\n",
    "\n",
    "\t\t'KMB','KIM','KMI','KLAC','KSS','KHC','KR','LB','LLL','LH','LRCX','LEG','LEN','LUK',\n",
    "\n",
    "\t\t'LLY','LNC','LKQ','LMT','L','LOW','LYB','MTB','MAC','M','MRO','MPC','MAR','MMC','MLM',\n",
    "\n",
    "\t\t'MAS','MA','MAT','MKC','MCD','MCK','MDT','MRK','MET','MTD','MGM','KORS','MCHP','MU',\n",
    "\n",
    "\t\t'MSFT','MAA','MHK','TAP','MDLZ','MON','MNST','MCO','MS','MOS','MSI','MYL','NDAQ',\n",
    "\n",
    "\t\t'NOV','NAVI','NTAP','NFLX','NWL','NFX','NEM','NWSA','NWS','NEE','NLSN','NKE','NI',\n",
    "\n",
    "\t\t'NBL','JWN','NSC','NTRS','NOC','NCLH','NRG','NUE','NVDA','ORLY','OXY','OMC','OKE',\n",
    "\n",
    "\t\t'ORCL','PCAR','PKG','PH','PDCO','PAYX','PYPL','PNR','PBCT','PEP','PKI','PRGO','PFE',\n",
    "\n",
    "\t\t'PCG','PM','PSX','PNW','PXD','PNC','RL','PPG','PPL','PX','PCLN','PFG','PG','PGR',\n",
    "\n",
    "\t\t'PLD','PRU','PEG','PSA','PHM','PVH','QRVO','PWR','QCOM','DGX','RRC','RJF','RTN','O',\n",
    "\n",
    "\t\t'RHT','REG','REGN','RF','RSG','RMD','RHI','ROK','COL','ROP','ROST','RCL','CRM','SBAC',\n",
    "\n",
    "\t\t'SCG','SLB','SNI','STX','SEE','SRE','SHW','SIG','SPG','SWKS','SLG','SNA','SO','LUV',\n",
    "\n",
    "\t\t'SPGI','SWK','SBUX','STT','SRCL','SYK','STI','SYMC','SYF','SNPS','SYY','TROW','TPR',\n",
    "\n",
    "\t\t'TGT','TEL','FTI','TXN','TXT','TMO','TIF','TWX','TJX','TMK','TSS','TSCO','TDG','TRV',\n",
    "\n",
    "\t\t'TRIP','FOXA','FOX','TSN','UDR','ULTA','USB','UAA','UA','UNP','UAL','UNH','UPS','URI',\n",
    "\n",
    "\t\t'UTX','UHS','UNM','VFC','VLO','VAR','VTR','VRSN','VRSK','VZ','VRTX','VIAB','V','VNO',\n",
    "\n",
    "\t\t'VMC','WMT','WBA','DIS','WM','WAT','WEC','WFC','HCN','WDC','WU','WRK','WY','WHR','WMB',\n",
    "\n",
    "\t\t'WLTW','WYN','WYNN','XEL','XRX','XLNX','XL','XYL','YUM','ZBH','ZION','ZTS']\n",
    "\n",
    "\t\t\n",
    "\n",
    "    bad_names =[] #to keep track of failed queries\n",
    "\n",
    "\n",
    "\n",
    "    \"\"\"here we use the concurrent.futures module's ThreadPoolExecutor\n",
    "        to speed up the downloads buy doing them in parallel \n",
    "        as opposed to sequentially \"\"\"\n",
    "\n",
    "    #set the maximum thread number\n",
    "    max_workers = 50\n",
    "    workers = min(max_workers, len(s_and_p)) #in case a smaller number of stocks than threads was passed in\n",
    "    with futures.ThreadPoolExecutor(workers) as executor:\n",
    "        res = executor.map(download_stock, s_and_p)\n",
    "\n",
    "\n",
    "    \"\"\" Save failed queries to a text file to retry \"\"\"\n",
    "    if len(bad_names) > 0:\n",
    "        with open('failed_queries.txt','w') as outfile:\n",
    "            for name in bad_names:\n",
    "                outfile.write(name+'\\n')\n",
    "\n",
    "    #timing:\n",
    "    finish_time = datetime.now()\n",
    "    duration = finish_time - now_time\n",
    "    minutes, seconds = divmod(duration.seconds, 60)\n",
    "    print(f'The threaded script took {minutes} minutes and {seconds} seconds to run.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
